package com.hhx.ec.utils;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;

import org.apache.commons.lang3.StringUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class WebmagicUtils implements PageProcessor{

    //设置爬虫参数
    private String baseUrl = "";
    private String spiderUrl = "";
    private String videoUrlTag = "";
    private String videoTitleTag = "";
    private String downloadVideoPath = "";

    public WebmagicUtils(String baseUrl,String spiderUrl,String videoUrlTag,
    		String videoTitleTag,String downloadVideoPath){
        this.baseUrl = baseUrl;
        this.spiderUrl = spiderUrl;
        this.videoUrlTag = videoUrlTag;
        this.videoTitleTag = videoTitleTag;
        this.downloadVideoPath = downloadVideoPath;
    }

  
	// 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private int spiderIntervalSecond = 2;
    private int retryTimes= 3;
    private Site site = Site.me().setRetryTimes(retryTimes).setSleepTime(spiderIntervalSecond)
                                  .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31")
                                  .setCharset("UTF-8");

    
    // process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
    public void process(Page page) {
        page.setCharset("utf-8");
        //从页面发现后续的url地址来抓取
        page.addTargetRequests(page.getHtml().links().regex(spiderUrl).all());
        if(page.getUrl().regex(spiderUrl).match()) {
            String videoUrl = page.getHtml().xpath(videoUrlTag).toString();
            if(StringUtils.isNotBlank(videoUrl)){
                if(!videoUrl.contains("http://")){
                    videoUrl = baseUrl + videoUrl;
                }
                String videoFormat =videoUrl.substring(videoUrl.lastIndexOf("."),videoUrl.length());
                String title = page.getHtml().xpath(videoTitleTag).toString();
                File destFile = new File(downloadVideoPath + title + videoFormat);
                downloadVideo(videoUrl, destFile);
            }
        }
    }

	@Override
	public Site getSite() {
		
		   return site;
	}
	
	 /**
     * 下载视频
     * @param url
     * @throws IOException
     */
    public static void downloadVideo(String url,File destFile)  {
        try {
            URL videoUrl = new URL(url);
            InputStream is = videoUrl.openStream();
            FileOutputStream fos = new FileOutputStream(destFile);

            int len = 0;
            byte[] buffer = new byte[1024];
            while ((-1) != (len = is.read(buffer))) {
                fos.write(buffer, 0, len);
            }
            fos.flush();

            if (null != fos) {
                fos.close();
            }

            if (null != is) {
                is.close();
            }
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
	
	

}
